package com.uusee.crawler.job.baike;

import java.io.InputStream;
import java.util.Properties;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.BaikeDataSourceDbWriter;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.baike.BaikeDataSourcePageProcessor;
import com.uusee.framework.bo.UniversalBo;
import com.uusee.shipshape.bk.model.BaikeDataSource;

public class ImdbCNBaikeFastCrawlJob {
	private static Log log = LogFactory.getLog(ImdbCNBaikeFastCrawlJob.class);
	private FetchHTTP fetch = new FetchHTTP();
	private BaikeDataSourcePageProcessor overviewPageProcessor;
	private BaikeDataSourceDbWriter dbWriter = new BaikeDataSourceDbWriter();
	
	private int start;
	private int end;

	public CrawlURI createCrawlURI(String oriId) {
		CrawlURI crawlURI = new CrawlURI();
		String crawlUrl = "http://www.imdb.cn/title/tt" + StringUtils.leftPad(oriId, 7, '0');
		crawlURI.setSourceSite("www.imdb.cn");
		crawlURI.setCrawlUrl(crawlUrl);
		crawlURI.setCharset("gb2312");
		return crawlURI;
	}
	public void doExecute() {
		try {
			for (int i = start; i < end; i++) {
				try {
					crawl(createCrawlURI(i+""));
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		} catch (Exception e) {
			log.error(e);
		}
	}
	public void crawl(CrawlURI crawlURI) {
		try {
			fetch.process(crawlURI);
			overviewPageProcessor.process(crawlURI);
			if (crawlURI.getCrawlStatus() == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				BaikeDataSource bds = (BaikeDataSource)crawlURI.getModel();
				
				dbWriter.process(crawlURI);
				Thread.sleep(3000);
			}
		} catch (Exception e) {
			e.printStackTrace();
			log.error(e);
		}
	}

	public static void main(String[] args) {
		try {
			
			ApplicationContext acx = new ClassPathXmlApplicationContext("spring/applicationContext-framework.xml");
			UniversalBo universalBo = (UniversalBo) acx.getBean("universalBo");
			BaikeDataSourceDbWriter dbWriter = new BaikeDataSourceDbWriter();
			dbWriter.setUniversalBo(universalBo);
			
			InputStream in = ClassLoader.getSystemResourceAsStream("regex/www.imdb.cn/overview.properties");
			Properties processPageRegex = new Properties();
			processPageRegex.load(in);
			BaikeDataSourcePageProcessor pageProcessor = new BaikeDataSourcePageProcessor(processPageRegex);
			
			ImdbCNBaikeFastCrawlJob job = new ImdbCNBaikeFastCrawlJob();
			job.dbWriter = dbWriter;
			job.overviewPageProcessor = pageProcessor;
			//finished:5000-60654
			job.start=60654;
			job.end=1000000;
			job.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}
